In [1]:
import itertools

In [9]:
import numpy as np

Figure out how to divide an iterable into evenly-sized chunks.


In [3]:
ids = ('C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9')

Suppose the iterable is a tuple of all possible pairs from ids:


In [4]:
pairs = tuple(itertools.combinations(ids, 2))

In [5]:
len(pairs)


Out[5]:
36

Also suppose that there are 16 processors, each operating on a single chunk.


In [6]:
numProc = 16

In [2]:
def get_chunk_lengths(vec, numProc):
    """Get the length of each chunk"""
    numDivFloor = len(vec)//numProc
    lengths = [numDivFloor]*numProc
    remainder = len(vec) - numDivFloor*numProc
    if remainder > len(vec):
        print('Error: remainder greater than vector length')
        return
    else:
        for i in range(remainder):
            lengths[i] += 1
        return lengths

In [7]:
splitlengths = get_chunk_lengths(pairs, numProc)

In [8]:
splitlengths


Out[8]:
[3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]

In [16]:
divpos = np.cumsum([0] + splitlengths)

In [17]:
divpos


Out[17]:
array([ 0,  3,  6,  9, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36])

In [18]:
len(divpos)


Out[18]:
17

In [22]:
def chunks(divpos, vec):
    for i in range(len(divpos)-1):
        yield vec[divpos[i]:divpos[i+1]]

In [23]:
tuple(chunks(divpos, pairs))


Out[23]:
((('C1', 'C2'), ('C1', 'C3'), ('C1', 'C4')),
 (('C1', 'C5'), ('C1', 'C6'), ('C1', 'C7')),
 (('C1', 'C8'), ('C1', 'C9'), ('C2', 'C3')),
 (('C2', 'C4'), ('C2', 'C5'), ('C2', 'C6')),
 (('C2', 'C7'), ('C2', 'C8')),
 (('C2', 'C9'), ('C3', 'C4')),
 (('C3', 'C5'), ('C3', 'C6')),
 (('C3', 'C7'), ('C3', 'C8')),
 (('C3', 'C9'), ('C4', 'C5')),
 (('C4', 'C6'), ('C4', 'C7')),
 (('C4', 'C8'), ('C4', 'C9')),
 (('C5', 'C6'), ('C5', 'C7')),
 (('C5', 'C8'), ('C5', 'C9')),
 (('C6', 'C7'), ('C6', 'C8')),
 (('C6', 'C9'), ('C7', 'C8')),
 (('C7', 'C9'), ('C8', 'C9')))

Try it on an iterable that is the range of integers from 10 to 74, with number of processors equal to 7:


In [28]:
numProc = 7
v = tuple(range(10,75))
splitLengths = get_chunk_lengths(v, numProc)
divpos = np.cumsum([0] + splitLengths)
tuple(chunks(divpos, v))


Out[28]:
((10, 11, 12, 13, 14, 15, 16, 17, 18, 19),
 (20, 21, 22, 23, 24, 25, 26, 27, 28, 29),
 (30, 31, 32, 33, 34, 35, 36, 37, 38),
 (39, 40, 41, 42, 43, 44, 45, 46, 47),
 (48, 49, 50, 51, 52, 53, 54, 55, 56),
 (57, 58, 59, 60, 61, 62, 63, 64, 65),
 (66, 67, 68, 69, 70, 71, 72, 73, 74))

In [ ]: